{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_csv(path = os.getcwd()):\n",
    "    df_list=[]\n",
    "    file_list = os.listdir(path)\n",
    "    file_list = [file for file in file_list if ('.csv' in file)]\n",
    "    \n",
    "    for file in file_list:\n",
    "        print(file)\n",
    "        \n",
    "        df = pd.read_csv(file, encoding='cp1252')\n",
    "        df = df.rename(columns={'Intermediate ': 'Intermediate'})\n",
    "        df['Topic'] = file[:-4]\n",
    "        df['Elementary'] = df['Elementary'].str.replace('\\n', ' ')\n",
    "        df['Intermediate'] = df['Intermediate'].str.replace('\\n', ' ')\n",
    "        df['Advanced'] = df['Advanced'].str.replace('\\n', ' ')\n",
    "        df_list.append(df[['Topic', 'Elementary', 'Intermediate', 'Advanced']])\n",
    "    \n",
    "    data = pd.concat(df_list)\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_txt():\n",
    "    path = '/Users/Jialin/Downloads/OneStopEnglishCorpus-master/Texts-SeparatedByReadingLevel/'\n",
    "    file_cat = ['Ele-Txt/', 'Adv-Txt/','Int-Txt/']\n",
    "    df_list=[]\n",
    "    \n",
    "    for cat in file_cat:\n",
    "        file_list = os.listdir(path + cat)\n",
    "        file_list = [file for file in file_list if ('.txt' in file)]\n",
    "\n",
    "        for file in file_list:\n",
    "            #print(file)\n",
    "            cate = file[-7:-4]\n",
    "            df = pd.read_table(path + cat + file, sep='\\n',header=None,names=['text'])\n",
    "            df['cat'] = cate\n",
    "            if ('WNL' in file[:-8]):\n",
    "                df['topic'] = file[4:-8]\n",
    "            else:\n",
    "                df['topic'] = file[:-8]\n",
    "            df_list.append(df)\n",
    "    \n",
    "    data = pd.concat(df_list)\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = process_txt()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0        SeaWorld\n",
       "0        Exercise\n",
       "0    On the trail\n",
       "0          Malala\n",
       "0       Novel way\n",
       "         ...     \n",
       "0       Meteorite\n",
       "0         Loterry\n",
       "0    India's rich\n",
       "0    Love hormone\n",
       "0      Can the US\n",
       "Name: topic, Length: 567, dtype: object"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.topic[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>cat</th>\n",
       "      <th>topic</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>SeaWorld’s profits fell by 84% and customers a...</td>\n",
       "      <td>ele</td>\n",
       "      <td>SeaWorld</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>The company teaches dolphins and killer whales...</td>\n",
       "      <td>ele</td>\n",
       "      <td>SeaWorld</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>It says fewer people are going to its parks an...</td>\n",
       "      <td>ele</td>\n",
       "      <td>SeaWorld</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>SeaWorld has been in the news since the 2013 d...</td>\n",
       "      <td>ele</td>\n",
       "      <td>SeaWorld</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>Animal rights organizations say that orcas kep...</td>\n",
       "      <td>ele</td>\n",
       "      <td>SeaWorld</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>SeaWorld has reduced ticket prices and spent $...</td>\n",
       "      <td>ele</td>\n",
       "      <td>SeaWorld</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>“We realize we have much work to do,” Manby sa...</td>\n",
       "      <td>ele</td>\n",
       "      <td>SeaWorld</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>“We will continue to fight with the facts beca...</td>\n",
       "      <td>ele</td>\n",
       "      <td>SeaWorld</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>Manby joined the company as CEO in 2015 to hel...</td>\n",
       "      <td>ele</td>\n",
       "      <td>SeaWorld</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>There are already plans for a new shark exhibi...</td>\n",
       "      <td>ele</td>\n",
       "      <td>SeaWorld</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>The company’s financial report from 6 August s...</td>\n",
       "      <td>ele</td>\n",
       "      <td>SeaWorld</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>People for the Ethical Treatment of Animals (P...</td>\n",
       "      <td>ele</td>\n",
       "      <td>SeaWorld</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>SeaWorld’s shares, which were worth $39 in 201...</td>\n",
       "      <td>ele</td>\n",
       "      <td>SeaWorld</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>Imagine that you read a headline 'Fit in four ...</td>\n",
       "      <td>ele</td>\n",
       "      <td>Exercise</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>Tabata remembers the first time he tested his ...</td>\n",
       "      <td>ele</td>\n",
       "      <td>Exercise</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>Tabata created his training programme after he...</td>\n",
       "      <td>ele</td>\n",
       "      <td>Exercise</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>The results were very surprising. After six we...</td>\n",
       "      <td>ele</td>\n",
       "      <td>Exercise</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>But you have to work very, very hard. You can’...</td>\n",
       "      <td>ele</td>\n",
       "      <td>Exercise</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>“Full effort at 170% of your VO 2 max is the b...</td>\n",
       "      <td>ele</td>\n",
       "      <td>Exercise</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>One person on an online forum wrote: “Most peo...</td>\n",
       "      <td>ele</td>\n",
       "      <td>Exercise</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 text  cat     topic\n",
       "0   SeaWorld’s profits fell by 84% and customers a...  ele  SeaWorld\n",
       "1   The company teaches dolphins and killer whales...  ele  SeaWorld\n",
       "2   It says fewer people are going to its parks an...  ele  SeaWorld\n",
       "3   SeaWorld has been in the news since the 2013 d...  ele  SeaWorld\n",
       "4   Animal rights organizations say that orcas kep...  ele  SeaWorld\n",
       "5   SeaWorld has reduced ticket prices and spent $...  ele  SeaWorld\n",
       "6   “We realize we have much work to do,” Manby sa...  ele  SeaWorld\n",
       "7   “We will continue to fight with the facts beca...  ele  SeaWorld\n",
       "8   Manby joined the company as CEO in 2015 to hel...  ele  SeaWorld\n",
       "9   There are already plans for a new shark exhibi...  ele  SeaWorld\n",
       "10  The company’s financial report from 6 August s...  ele  SeaWorld\n",
       "11  People for the Ethical Treatment of Animals (P...  ele  SeaWorld\n",
       "12  SeaWorld’s shares, which were worth $39 in 201...  ele  SeaWorld\n",
       "0   Imagine that you read a headline 'Fit in four ...  ele  Exercise\n",
       "1   Tabata remembers the first time he tested his ...  ele  Exercise\n",
       "2   Tabata created his training programme after he...  ele  Exercise\n",
       "3   The results were very surprising. After six we...  ele  Exercise\n",
       "4   But you have to work very, very hard. You can’...  ele  Exercise\n",
       "5   “Full effort at 170% of your VO 2 max is the b...  ele  Exercise\n",
       "6   One person on an online forum wrote: “Most peo...  ele  Exercise"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.to_csv('dataset.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
